# -*- coding: utf-8 -*-
import datetime
import socket
from urllib.parse import urljoin

from scrapy import Request
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose
from scrapy.spiders import Spider

from mansite.items import MansiteItem


class PerfectSpider(Spider):
    name = 'perfect'

    allowed_domains = ['m.nanrenvip.cc/perfect']
    start_urls = [
        # 'http://m.nanrenvip.cc/perfect/'
        'http://m.nanrenvip.cc/s1s1s1',
        'http://m.nanrenvip.cc/ideapocket'
    ]

    def parse(self, response):
        next_urls = response.xpath('//span[contains(@class,"right")]/a/@href').extract()
        next_parse_url = urljoin(response.url, next_urls[0])  # 爬取下一页的链接
        # if len(next_urls) > 1 and next_urls[0] != next_urls[1]:
        if response.url != next_parse_url:
            yield Request(next_parse_url, callback=self.parse, dont_filter=True)

        selectors = response.xpath('//li[@class="book-li"]')
        self.log('response %s return record items: %d' % (response.url, len(selectors)))
        for selector in selectors:
            yield self.parse_item(selector, response)

    def parse_item(self, selector, response):

        # 去除文本前后空格
        strip_fun = lambda i: i.strip()
        # 连接两个url
        joinurl_fun = lambda i: urljoin(response.url, i)

        ld = ItemLoader(item=MansiteItem(), selector=selector)

        # 输出到Item
        ld.add_xpath('title', './/h4[@class="book-title"]/text()', MapCompose(strip_fun))
        ld.add_xpath('desc', './/p[@class="book-desc"]/text()', MapCompose(strip_fun))
        ld.add_xpath('image', './/a/img/@data-echo', MapCompose(joinurl_fun))
        ld.add_xpath('duration', './/span/em[1]/text()', MapCompose(strip_fun))
        ld.add_xpath('type', './/span/em[2]/text()', MapCompose(strip_fun))
        ld.add_xpath('pubDate', './/span/em[3]/text()', MapCompose(strip_fun))
        ld.add_xpath('detailUrl', './/a[@class="book-layout"]/@href', MapCompose(joinurl_fun))

        ld.add_value('url', response.url)
        ld.add_value('server', socket.gethostname())
        ld.add_value('project', self.settings.get('BOT_NAME'))
        ld.add_value('timestamp', datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))

        item = ld.load_item()
        self.log('parse item for return %s' % item)
        return item
